{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Configs\n",
    "\n",
    "This code converts a Middle-format dataset to a ConKADI-format dataset. So before runinning this code, please prepare a Middle-format dataset.\n",
    "\n",
    "All you need to do is configuring the following 3 terms. Due to the randomness, the generated dataset may have some monior difference."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "ename": "SyntaxError",
     "evalue": "invalid syntax (<ipython-input-5-0ef69bbb209b>, line 3)",
     "output_type": "error",
     "traceback": [
      "\u001b[0;36m  File \u001b[0;32m\"<ipython-input-5-0ef69bbb209b>\"\u001b[0;36m, line \u001b[0;32m3\u001b[0m\n\u001b[0;31m    For Chinese Weibo\u001b[0m\n\u001b[0m              ^\u001b[0m\n\u001b[0;31mSyntaxError\u001b[0m\u001b[0;31m:\u001b[0m invalid syntax\n"
     ]
    }
   ],
   "source": [
    "# CCM source file path\n",
    "\n",
    "For Chinese Weibo\n",
    "# See the Step 1\n",
    "ccm_path = 'ccm/'\n",
    "# Traget Folder\n",
    "target_folder = 'base_data/ConKADI/'\n",
    "vocab_size = 50000\n",
    "# See the Step 1\n",
    "file_map = {\n",
    "    'train':'trainset.txt',\n",
    "    'dev':'validset.txt',\n",
    "    'test':'testset.txt',\n",
    "     'demo':'testset.txt'\n",
    "}\n",
    "\n",
    "# For English Reddit\n",
    "\n",
    "# ccm_path = 'ccm_py/data/'\n",
    "# target_folder = 'base_data/ConKADI_EN/'\n",
    "# vocab_size = 30000\n",
    "\n",
    "\n",
    "# file_map = {\n",
    "#     'train':'trainset_small.txt',\n",
    "#     'dev':'validset_small.txt',\n",
    "#     'test':'testset_small.txt',\n",
    "#      'demo':'testset_small.txt'\n",
    "# }\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Format check"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "#Word:30000  #Entity:27066\n",
      "Good!\n"
     ]
    }
   ],
   "source": [
    "import json\n",
    "data_train = []\n",
    "with open(ccm_path + file_map['train']) as f:\n",
    "    for idx, line in enumerate(f):\n",
    "        data_train.append(json.loads(line))\n",
    "        if len(data_train) % 1000 == 0: \n",
    "            break\n",
    "            \n",
    "with open(ccm_path + 'resource.txt') as f:\n",
    "        d = json.loads(f.readline())\n",
    "        csk_triples = d['csk_triples']\n",
    "        csk_entities = d['csk_entities']\n",
    "        raw_vocab = d['vocab_dict']\n",
    "        kb_dict = d['dict_csk']\n",
    "\n",
    "entity_vocab = set()\n",
    "with open(ccm_path + 'entity.txt',encoding='utf-8') as f:\n",
    "    for idx, line in enumerate(f):\n",
    "        entity = line.strip('\\n')\n",
    "        entity_vocab.add(entity)\n",
    "        \n",
    "word_vocab = set(raw_vocab)\n",
    "entity_full_vocab = set(csk_entities)\n",
    "print('#Word:%d  #Entity:%d' % (len(word_vocab), len(entity_vocab)))\n",
    "\n",
    "for my_case in data_train:\n",
    "    flag = False\n",
    "    for word,idx in zip(my_case['response'],my_case['response_triples']):\n",
    "        if word in entity_vocab:\n",
    "            flag = True\n",
    "        if idx != -1: \n",
    "            assert word in entity_full_vocab, word\n",
    "#     assert flag is True, flag \n",
    "    \n",
    "    flag = False\n",
    "    for word,idx in zip(my_case['post'],my_case['post_triples']):     \n",
    "        if word in entity_vocab:\n",
    "            flag = True\n",
    "        if idx != 0:\n",
    "            assert word in entity_full_vocab , word\n",
    "#     assert flag is True\n",
    "\n",
    "print('Good!')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create & Load Vocabs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "valid_vocab = set()\n",
    "with open(target_folder + 'uni_vocab.txt','w+',encoding='utf-8') as funi:\n",
    "    valid_vocab = set()\n",
    "    with open(ccm_path +'resource.txt') as f:\n",
    "        d = json.loads(f.readline())\n",
    "        csk_triples = d['csk_triples']\n",
    "        csk_entities = d['csk_entities']\n",
    "        raw_vocab = d['vocab_dict']\n",
    "        with open(target_folder + 'vocab.txt','w+',encoding='utf-8') as fsrc:\n",
    "                fsrc.write('<unk>\\n<s>\\n</s>\\n')\n",
    "                funi.write('<unk>\\n<s>\\n</s>\\n')\n",
    "                for word in  sorted(raw_vocab, key=raw_vocab.get, reverse=True)[0:vocab_size]:\n",
    "                    fsrc.write('%s\\n' % word)\n",
    "                    funi.write('%s\\n' % word)\n",
    "                    valid_vocab.add(word)\n",
    "    # Entity Vocab\n",
    "    with open(ccm_path +'entity.txt',encoding='utf-8') as f:\n",
    "        entity_list = []\n",
    "        for idx, line in enumerate(f):\n",
    "            entity = line.strip('\\n')\n",
    "            entity_list.append(entity)\n",
    "        with open(target_folder + 'entity_vocab.txt','w+',encoding='utf-8') as fsrc:\n",
    "                fsrc.write('#UNK\\n#N\\n#PE\\n#NH\\n#NT\\n')\n",
    "                funi.write('#UNK\\n#N\\n#PE\\n#NH\\n#NT\\n')\n",
    "                for word in  entity_list:\n",
    "                    fsrc.write('%s\\n' % word)\n",
    "                    if word not in valid_vocab:\n",
    "                        valid_vocab.add(word)\n",
    "                        funi.write('%s\\n' % word)\n",
    "\n",
    "    # Relation Vocab\n",
    "    with open(ccm_path +'relation.txt',encoding='utf-8') as f:\n",
    "        entity_list = []\n",
    "        for idx, line in enumerate(f):\n",
    "            entity = line.strip('\\n')\n",
    "            entity_list.append(entity)\n",
    "        with open(target_folder + 'relation_vocab.txt','w+',encoding='utf-8') as fsrc:\n",
    "                fsrc.write('#NF\\n#PR\\n#NR\\n')\n",
    "                for word in  entity_list:\n",
    "                    fsrc.write('%s\\n' % word)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create Dialogue Query-Response Pairs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import defaultdict\n",
    "length_counter = defaultdict(int)\n",
    "# Word_Level\n",
    "for key in file_map:\n",
    "    source_path = file_map[key]\n",
    "    with open(ccm_path +source_path) as f:\n",
    "        with open(target_folder+key+'.src','w+',encoding='utf-8') as fsrc:\n",
    "             with open(target_folder+key+'.tgt','w+',encoding='utf-8') as ftgt:\n",
    "                    for fidx, line in enumerate(f):\n",
    "                        if key == 'demo' and fidx > 999:\n",
    "                            break\n",
    "                        item = json.loads(line)\n",
    "                        message = item['post']\n",
    "                        response = item['response']\n",
    "                        fsrc.write('%s\\n' % ' '.join(message))\n",
    "                        ftgt.write('%s\\n' % ' '.join(response))\n",
    "                        length_counter[len(message)] += 1\n",
    "                        length_counter[len(response)] += 1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Generate Others"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'target_folder' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-4-33eab97c9841>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     11\u001b[0m         \u001b[0mstopwords\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mword\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     12\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 13\u001b[0;31m \u001b[0;32mwith\u001b[0m \u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtarget_folder\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0;34m'uni_vocab.txt'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mencoding\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'utf-8'\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mfuni\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     14\u001b[0m     \u001b[0;31m# Mask top 500\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     15\u001b[0m     \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m500\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'target_folder' is not defined"
     ]
    }
   ],
   "source": [
    "from collections import defaultdict\n",
    "import random\n",
    "random.seed(12345)\n",
    "fact_dict = dict()\n",
    "reverse_fact_dict = dict()\n",
    "prefix = 'naf'\n",
    "with open('stopwords.txt') as fin:\n",
    "    stopwords = dict()\n",
    "    for line in fin.readlines():\n",
    "        word = line.strip('\\n')\n",
    "        stopwords[word] = 1\n",
    "        \n",
    "with open(target_folder+'uni_vocab.txt',encoding='utf-8') as funi:\n",
    "    # Mask top 500\n",
    "    for i in range(500):\n",
    "        word = funi.readline().strip('\\n')\n",
    "        stopwords[word] = 2\n",
    "        \n",
    "with open(ccm_path +'resource.txt') as f:\n",
    "    d = json.loads(f.readline())\n",
    "    csk_triples = d['csk_triples']\n",
    "    csk_entities = d['csk_entities']\n",
    "    raw_vocab = d['vocab_dict']\n",
    "\n",
    "max_golden_len = 0\n",
    "max_fact_token_num = 0\n",
    "\n",
    "\n",
    "for key in file_map:\n",
    "    source_path = file_map[key]\n",
    "    with open(ccm_path +source_path) as f:\n",
    "        with open(target_folder+key+'.%s_fact' % prefix,'w+',encoding='utf-8') as fsrc:\n",
    "            with open(target_folder+key+'.%s_golden_fact' % prefix,'w+',encoding='utf-8') as fgd:\n",
    "                 with open(target_folder+key+'.%s_golden_facts' % prefix,'w+',encoding='utf-8') as fgds:\n",
    "                    with open(target_folder+key+'.tgt.%s_ent' % prefix,'w+',encoding='utf-8') as fent:\n",
    "                         with open(target_folder+key+'.tgt.%s_ment' % prefix,'w+',encoding='utf-8') as fment:\n",
    "                            with open(target_folder+key+'.tgt.%s_cp' % prefix,'w+',encoding='utf-8') as fcp:\n",
    "                                with open(target_folder+key+'.tgt.%s_entcp' % prefix,'w+',encoding='utf-8') as fec:\n",
    "                                    for fidx, line in enumerate(f):\n",
    "                                        \n",
    "                                        if key == 'demo' and fidx > 2999:\n",
    "                                            break\n",
    "                                            \n",
    "                                        my_case = json.loads(line)\n",
    "                                        golden_et = [] # golden entities\n",
    "                                        et = [] #all entities\n",
    "                                        \n",
    "                                        golden_triplet = set()\n",
    "                                        for word,idx in zip(my_case['response'],my_case['response_triples']):\n",
    "                                            if idx != -1:\n",
    "                                                golden_triplet.add(idx)\n",
    "                                                \n",
    "                                        # NOT A FACT\n",
    "                                        fact = ['#N','#N','#NH','#NF','#NT']\n",
    "                                        ef = ' '.join(fact)\n",
    "                                        if ef not in fact_dict:\n",
    "                                            fact_dict[ef] = len(fact_dict)\n",
    "                                            ef_id = fact_dict[ef]\n",
    "                                            reverse_fact_dict[ef_id] = ef\n",
    "                                        ef_id = str(fact_dict[ef])\n",
    "                                        et.append(ef_id)\n",
    "\n",
    "                                        position_dict = dict()\n",
    "\n",
    "                                        for entity,triple in zip(my_case['all_entities'], my_case['all_triples']):\n",
    "                                            for x,y in zip(entity,triple):\n",
    "                                                ent = '%s' % csk_entities[x]\n",
    "                                                fact = csk_triples[y].split(',')\n",
    "                                                for i in range(3):\n",
    "                                                    fact[i] = fact[i].strip(' ')\n",
    "                                                entity_in_post = fact[0] if fact[0] != ent else fact[2]\n",
    "                                                ef = ' '.join([entity_in_post, ent]+fact)\n",
    "                                                if ef not in fact_dict:\n",
    "                                                    fact_dict[ef] = len(fact_dict)\n",
    "                                                    ef_id = fact_dict[ef]\n",
    "                                                    reverse_fact_dict[ef_id] = ef\n",
    "                                                ef_id = str(fact_dict[ef])\n",
    "                                                if y in golden_triplet:\n",
    "                                                    golden_et.append(ef_id)\n",
    "                                                position_dict[y] = len(et)\n",
    "                                                et.append(ef_id)\n",
    "                              \n",
    "                                        ent_response = []\n",
    "                                        cp_response = []\n",
    "                                        entcp_response = []\n",
    "                                        copy_position_dict = dict()\n",
    "                                        for i, post_word in enumerate(my_case['post']):\n",
    "                                            if i > 49:\n",
    "                                                break\n",
    "                                            if post_word not in stopwords:\n",
    "                                                copy_position_dict[post_word] = i\n",
    "\n",
    "                                        for word,idx in zip(my_case['response'],my_case['response_triples']):\n",
    "                                            if idx != -1 and idx in position_dict:\n",
    "                                                entcp_response.append('$ENT_%d' % (position_dict[idx]))\n",
    "                                                ent_response.append('$ENT_%d' % (position_dict[idx]))\n",
    "                                                if max(max_fact_token_num, position_dict[idx]) > max_fact_token_num:\n",
    "                                                    max_fact_token_num = max(max_fact_token_num, position_dict[idx]) \n",
    "                                                    print('max_fact_num %d' % max_fact_token_num)\n",
    "                                                #ent_response.append('$ENT_%d_[%s]' % (position_dict[idx],word+'/'+reverse_fact_dict[int(et[position_dict[idx]])] ))\n",
    "                                            elif word in copy_position_dict:\n",
    "                                                entcp_response.append('$CP_%d' % (copy_position_dict[word]))\n",
    "                                                ent_response.append(word)\n",
    "                                            else:\n",
    "                                                entcp_response.append(word)\n",
    "                                                ent_response.append(word)\n",
    "\n",
    "                                            if word in copy_position_dict:\n",
    "                                                cp_response.append('$CP_%d' % (copy_position_dict[word])) \n",
    "                                            else:\n",
    "                                                cp_response.append(word)\n",
    "                                        fent.write('%s\\n' % ' '.join(ent_response))\n",
    "                                        fcp.write('%s\\n' % ' '.join(cp_response))\n",
    "                                        fec.write('%s\\n' % ' '.join(entcp_response))\n",
    "                                        fsrc.write('%s\\n' % ' '.join(et))\n",
    "                                        if len(golden_et) == 0:\n",
    "                                            golden_et.append(str(0))\n",
    "                                        fgds.write('%s\\n' % ' '.join(golden_et))\n",
    "                                        golden_et = random.sample(golden_et, 1)\n",
    "                                        fgd.write('%s\\n' % ' '.join(golden_et))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(target_folder+'entity_vocab.txt',encoding='utf-8') as f:\n",
    "    entity_words = [x.strip() for x in f.readlines()]\n",
    "\n",
    "entity_dict = dict()\n",
    "for entity in entity_words:\n",
    "    entity_dict[entity] = len(entity_dict)\n",
    "\n",
    "with open(target_folder+'vocab.txt',encoding='utf-8') as f:\n",
    "    vocabs = [x.strip() for x in f.readlines()]\n",
    "\n",
    "vocab_dict = dict(zip(vocabs, range(len(vocabs))))\n",
    "\n",
    "with open(target_folder+'word2entity.txt','w+',encoding='utf-8') as f:\n",
    "    for word in vocabs:\n",
    "        if word not in entity_dict:\n",
    "            f.write('0\\n')\n",
    "        else:\n",
    "            f.write('%d\\n' % entity_dict[word])\n",
    "            \n",
    "with open(target_folder+'entity2word.txt','w+',encoding='utf-8') as f:\n",
    "    for entity in entity_words:\n",
    "        if entity not in vocab_dict:\n",
    "            f.write('0\\n')\n",
    "        else:\n",
    "            f.write('%d\\n' % vocab_dict[entity])\n",
    "\n",
    "\n",
    "with open(target_folder+'%s_cpent_vocab.txt' % prefix,'w+',encoding='utf-8') as f:\n",
    "    for word in vocabs:\n",
    "        f.write('%s\\n' % word)\n",
    "    for i in range(35):\n",
    "        f.write('$CP_%d\\n' % i)\n",
    "    for i in range(305):\n",
    "        f.write('$ENT_%d\\n' % i)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'reverse_fact_dict' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-3-07caa7828557>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mreverse_fact_dict\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0;32mwith\u001b[0m \u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtarget_folder\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0;34m'%s_fact_vocab.txt'\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0mprefix\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m'w+'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mencoding\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'utf-8'\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m     \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mreverse_fact_dict\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m         \u001b[0mitems\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mreverse_fact_dict\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msplit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mi\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m0\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mitems\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0mitems\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mitems\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0mitems\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m4\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'reverse_fact_dict' is not defined"
     ]
    }
   ],
   "source": [
    "print(reverse_fact_dict[1])\n",
    "with open(target_folder+'%s_fact_vocab.txt' % prefix,'w+', encoding='utf-8') as f:\n",
    "    for i in range(len(reverse_fact_dict)):\n",
    "        items = reverse_fact_dict[i].split()\n",
    "        if i == 0 or (items[0] == items[2] and items[1] == items[4]):\n",
    "            f.write('%s\\n' % (reverse_fact_dict[i]))\n",
    "        else:\n",
    "            items[2] = items[0]\n",
    "            items[4] = items[1]\n",
    "            items[3] = items[3]\n",
    "            f.write('%s\\n' % (' '.join(items)))\n",
    "fact_dict =None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'target_folder' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-2-2a188eacab51>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0mfact_vocab\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0;32mwith\u001b[0m \u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtarget_folder\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0;34m'%s_fact_vocab.txt'\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0mprefix\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mencoding\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'utf-8'\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      3\u001b[0m     \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mline\u001b[0m \u001b[0;32min\u001b[0m \u001b[0menumerate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreadlines\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m         \u001b[0mtgt_word\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mline\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msplit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m         \u001b[0mfact_vocab\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtgt_word\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'target_folder' is not defined"
     ]
    }
   ],
   "source": [
    "fact_vocab = []\n",
    "with open(target_folder+'%s_fact_vocab.txt' % prefix, encoding='utf-8') as f:\n",
    "    for i, line in enumerate(f.readlines()):\n",
    "        tgt_word = line.split()[1]\n",
    "        fact_vocab.append(tgt_word)\n",
    "\n",
    "print(fact_vocab[0:100])\n",
    "\n",
    "for fact_vocab_size in [0]:\n",
    "    tgt_prefx = prefix\n",
    "    print(tgt_prefx)\n",
    "    for key in file_map:\n",
    "        source_path = file_map[key]\n",
    "        with open(ccm_path+source_path) as f:\n",
    "            with open(target_folder+key+'.%s_fact' % tgt_prefx,encoding='utf-8') as fsrc:\n",
    "                with open(target_folder+key+'.%s_golden_fact' % tgt_prefx,encoding='utf-8') as fgd:\n",
    "                     with open(target_folder+key+'.%s_golden_fact_position' % tgt_prefx, 'w+', encoding='utf-8') as frgd:\n",
    "                            with open(target_folder+key+'.%s_neg_fact_position' % tgt_prefx, 'w+', encoding='utf-8') as fneg:\n",
    "                                facts = [ x.strip('\\n').split() for x in fsrc.readlines()]\n",
    "                                golden_fact = [x.strip('\\n') for x in fgd.readlines()]\n",
    "                                for fact, golden in zip(facts, golden_fact):\n",
    "                                    golden_index = fact.index(golden)\n",
    "                                    neg_index = []\n",
    "                                    for x in range(len(fact)):\n",
    "                                        if fact_vocab[x] != fact_vocab[golden_index]: \n",
    "                                            neg_index.append(str(x))\n",
    "    #                                     else:\n",
    "    #                                         neg_index.append(\"XXXX\"+str(x))\n",
    "                                    frgd.write('%s\\n' % golden_index)\n",
    "                                    fneg.write('%s\\n' % ' '.join(neg_index))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'target_folder' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-1-0bff4a27336a>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0mprefix\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m'naf'\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0mfact_vocab\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0;32mwith\u001b[0m \u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtarget_folder\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0;34m'%s_fact_vocab.txt'\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0mprefix\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mencoding\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'utf-8'\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      4\u001b[0m     \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mline\u001b[0m \u001b[0;32min\u001b[0m \u001b[0menumerate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreadlines\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m         \u001b[0mtgt_word\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mline\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msplit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'target_folder' is not defined"
     ]
    }
   ],
   "source": [
    "\n",
    "prefix = 'naf'\n",
    "fact_vocab = []\n",
    "with open(target_folder+'%s_fact_vocab.txt' % prefix, encoding='utf-8') as f:\n",
    "    for i, line in enumerate(f.readlines()):\n",
    "        tgt_word = line.split()[1]\n",
    "        fact_vocab.append(tgt_word)\n",
    "\n",
    "print(fact_vocab[0:100])\n",
    "for fact_vocab_size in [-1]:\n",
    "    tgt_prefx = prefix\n",
    "    print(tgt_prefx)\n",
    "    for key in file_map:\n",
    "        source_path = file_map[key]\n",
    "        with open(ccm_path + source_path) as f:\n",
    "            with open(target_folder+key+'.%s_fact' % tgt_prefx,encoding='utf-8') as fsrc:\n",
    "                with open(target_folder+key+'.%s_golden_facts' % tgt_prefx,encoding='utf-8') as fgd:\n",
    "                    with open(target_folder+key+'.%s_golden_facts_position' % tgt_prefx, 'w+', encoding='utf-8') as frgd:\n",
    "                            facts = [ x.strip('\\n').split() for x in fsrc.readlines()]\n",
    "                            golden_facts = [x.strip('\\n').split() for x in fgd.readlines()]\n",
    "                            for fact, golden in zip(facts, golden_facts):\n",
    "                                golden_index = set([str(fact.index(x)) for x in golden])\n",
    "                                golden_index = ' '.join(golden_index)\n",
    "                                frgd.write('%s\\n' % golden_index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
